KDE plot example:


In [75]:
%matplotlib inline
import numpy as np
import pandas as pd
import seaborn as sns
import os
import pylab
import matplotlib.pyplot as plt

#sql stuff:
from sqlalchemy import create_engine
from sqlalchemy_utils import database_exists, create_database
import psycopg2

from bokeh.io import output_notebook

%config InlineBackend.figure_format = 'retina'

In [76]:
output_notebook()


Loading BokehJS ...

In [2]:
dbname = 'airbnb_db'
username = 'alexpapiu'

con = psycopg2.connect(database = dbname, user = username)

In [3]:
nbd = "East Village"

#Private Room or Entire home/apt
#room_type = "Entire home/apt"
room_type = "Private room"

In [4]:
train = pd.read_sql_query("""
                          SELECT * FROM small_listings 
                          WHERE neighbourhood_cleansed = %(nbd)s AND
                          room_type = %(room_type)s;
                          """,
                          con, index_col = "id",
                          params = {"nbd":nbd, "room_type":room_type})

median_price = train["price"].mean()

In [5]:
"95% of the listings are in between ${0} and ${1}".format(train["price"].quantile(0.05),train["price"].quantile(0.96))


Out[5]:
'95% of the listings are in between $60.0 and $200.0'

In [6]:
sns.kdeplot(train["price"].values, bw=25.0)
plt.axvline(x=median_price)
plt.axvline(train["price"].quantile(0.05), color = "red", linewidth = 1)
plt.axvline(train["price"].quantile(0.95), color = "red", linewidth = 1)

#plt.text(median_price + 50,0.01,'The Median Price is {0}'.format(np.round(median_price, 1)), fontsize=12)
plt.title('The Median Price is {0}'.format(np.round(median_price, 1)))


if room_type == "Entire home/apt":
    plt.xlim([0, 500])
else:
    plt.xlim([0, 350])


//anaconda/lib/python3.5/site-packages/statsmodels/nonparametric/kdetools.py:20: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future
  y = X[:m/2+1] + np.r_[0,X[m/2+1:],0]*1j

Prediction Modelling:


In [7]:
train = pd.read_sql_query("select * from listings_price", con)

In [8]:
train


Out[8]:
id listing_url scrape_id last_scraped name summary space description experiences_offered neighborhood_overview ... license jurisdiction_names instant_bookable cancellation_policy require_guest_profile_picture require_guest_phone_verification calculated_host_listings_count reviews_per_month preds diff
0 685006 https://www.airbnb.com/rooms/685006 20161203092247 2016-12-04 Best Studio on Prospect Park Perfect studio for couples or individuals look... THE SPACE: The apartment is a good sized studi... Perfect studio for couples or individuals look... none The apartment is right across the street from ... ... None None f strict f f 1 2.87 78.206529 3.206529
1 9461238 https://www.airbnb.com/rooms/9461238 20161203092247 2016-12-04 1 bedrms Williamsburg 1st Stop L Very well located 1 bedroom apt on the South o... The apartment is yours for your stay; it has t... Very well located 1 bedroom apt on the South o... none Prime Williamsburg, the hot South side: Steps ... ... None None f strict f f 1 4.27 174.554158 57.554158
2 4873690 https://www.airbnb.com/rooms/4873690 20161203092247 2016-12-04 Private room, 10min to Times Square This 2br apartment in LIC offers a private bed... UPDATE: My apartment was recently featured in ... This 2br apartment in LIC offers a private bed... none Long Island City is the ultimate, in my mind, ... ... None None f flexible f f 1 1.81 91.726522 2.726522
3 12738047 https://www.airbnb.com/rooms/12738047 20161203092247 2016-12-04 clean and tidy separated livingroom It is the living room separated in an 2b1b apa... None It is the living room separated in an 2b1b apa... none None ... None None f moderate f f 1 NaN 99.614082 19.614082
4 15359479 https://www.airbnb.com/rooms/15359479 20161203092247 2016-12-04 Awesome One Bedroom Gramercy Park Dear Guest, Welcome to New York! Thanks for vi... [Details] I'll let the pictures speak for the... Dear Guest, Welcome to New York! Thanks for vi... none None ... None None f flexible f f 1 1.00 209.892523 9.892523
5 14179829 https://www.airbnb.com/rooms/14179829 20161203092247 2016-12-04 HUGE zen room, queen bed in beautiful Sunnyside Huge bedroom the size of a studio apartment wi... Very zen, beautifully decorated, perfect for m... Huge bedroom the size of a studio apartment wi... none Located on a beautiful, quiet tree lined stree... ... None None f moderate f f 1 NaN 36.585302 -26.414698
6 810483 https://www.airbnb.com/rooms/810483 20161203092247 2016-12-04 Beautiful, central,sunny, quiet shared apt This apartment is located in the heart of the ... Clean and modern, with a renovated kitchen and... This apartment is located in the heart of the ... none None ... None None f strict f f 1 0.02 109.078367 -9.921633
7 9854420 https://www.airbnb.com/rooms/9854420 20161203092247 2016-12-04 Beautiful one bedroom in Soho The apartment is on King street, close to Soho... None The apartment is on King street, close to Soho... none None ... None None f flexible f f 1 1.42 212.590218 12.590218
8 4349358 https://www.airbnb.com/rooms/4349358 20161203092247 2016-12-04 Spacious UWS Apt. by Central Park Enjoy your stay in a spacious, beautiful brown... Come experience the rhythm and rhapsody of New... Enjoy your stay in a spacious, beautiful brown... none The apartment is located in a beautiful, safe,... ... None None f strict f f 1 3.77 122.546415 -2.453585
9 9330135 https://www.airbnb.com/rooms/9330135 20161203092247 2016-12-04 Brand new 1 BR apt Steps from train Brand new building with modern decor Large apa... None Brand new building with modern decor Large apa... none None ... None None f flexible f f 1 NaN 131.160573 46.160573
10 14310864 https://www.airbnb.com/rooms/14310864 20161203092247 2016-12-04 Spacious Studio Near Prospect Park! Spacious, private studio available - A/C inclu... None Spacious, private studio available - A/C inclu... none None ... None None f flexible f f 1 NaN 47.252210 -32.747790
11 1465807 https://www.airbnb.com/rooms/1465807 20161203092247 2016-12-04 Park Views! Upper Manhattan 1BR AVAILABLE JUNE 24, 2015 THROUGH JULY 2, 2015 O... AVAILABLE JUNE 24, 2015 THROUGH JULY 2, 2015 O... none None ... None None f flexible f f 1 NaN 135.153945 56.153945
12 11165126 https://www.airbnb.com/rooms/11165126 20161203092247 2016-12-04 The Best Of UNION SQUARE & CHELSEA Centrally located between Union Square & Chels... This beautiful modern 725 sq ft. pre-war apart... Centrally located between Union Square & Chels... none Architecture, art, and fine dining meets a thr... ... None None f strict f f 1 1.99 209.344142 -35.655858
13 16014791 https://www.airbnb.com/rooms/16014791 20161203092247 2016-12-04 Nice clean style apt. Central midtwn near Javits My place is near Highland Park, Javits, Time S... The living room have a beautiful designer comf... My place is near Highland Park, Javits, Time S... none I love the convenient of walking distance to e... ... None None t strict f f 1 4.00 206.863576 96.863576
14 16145655 https://www.airbnb.com/rooms/16145655 20161203092247 2016-12-04 Spacious 1-bd, sleeps 4, 20 min to Times Square Full access to large 1-bedroom apartment that ... None Full access to large 1-bedroom apartment that ... none None ... None None f flexible f f 1 NaN 147.098114 -2.901886
15 7344066 https://www.airbnb.com/rooms/7344066 20161203092247 2016-12-04 Gorgeous West Village 1-bdm apt This gorgeous 1-bedroom apt is located in the ... None This gorgeous 1-bedroom apt is located in the ... none None ... None None t strict f f 1 1.19 192.251792 -57.748208
16 9461187 https://www.airbnb.com/rooms/9461187 20161203092247 2016-12-04 Apartment 20 minutes from Manhattan Quick access to midtown Manhattan (Times Squar... None Quick access to midtown Manhattan (Times Squar... none None ... None None f flexible f f 1 NaN 155.463699 12.463699
17 15752636 https://www.airbnb.com/rooms/15752636 20161203092247 2016-12-04 Spacious Private Room in Brooklyn Located in a lively neighbourhood in Bushwick,... None Located in a lively neighbourhood in Bushwick,... none None ... None None f flexible f f 1 NaN 36.450209 -3.549791
18 1727923 https://www.airbnb.com/rooms/1727923 20161203092247 2016-12-04 Large private 1BR with backyard - Williamsburg Quiet, comfortable and 100% private apartment ... Beautiful newly renovated 1 bedroom studio in ... Quiet, comfortable and 100% private apartment ... none Williamsburg is the most dynamic neighborhood ... ... None None f flexible f f 1 0.16 150.209761 52.209761
19 11296173 https://www.airbnb.com/rooms/11296173 20161203092247 2016-12-04 Lovely NYC Apartment July 16-30 A light filled apartment on the 20th floor wit... None A light filled apartment on the 20th floor wit... none None ... None None f flexible f f 1 NaN 227.316172 -172.683828
20 7364008 https://www.airbnb.com/rooms/7364008 20161203092247 2016-12-04 Bohéme 1BR steps from Prospect Park Beautiful, spacious 1 BR with 5 minutes walk t... This place was a labor of love! At this point,... Beautiful, spacious 1 BR with 5 minutes walk t... none None ... None None f strict f f 1 0.12 122.746232 23.746232
21 15097255 https://www.airbnb.com/rooms/15097255 20161203092247 2016-12-04 Luxury 1 Br in Luxury Doorman Building My place is close to Brooklyn Bridge, Manhatta... The apartment is located in a luxury high rise... My place is close to Brooklyn Bridge, Manhatta... none None ... None None f flexible f f 1 NaN 144.980673 -45.019327
22 3562906 https://www.airbnb.com/rooms/3562906 20161203092247 2016-12-04 HighRise Studio-Walk 2 Times Square Share my cozy Midtown West space in the heart ... This is a spacious studio apartment in Midtown... Share my cozy Midtown West space in the heart ... none This is centrally located in midtown. Times Sq... ... None None f strict f f 1 0.62 117.064227 17.064227
23 8699290 https://www.airbnb.com/rooms/8699290 20161203092247 2016-12-04 Private Room Great Area Great Neighborhood. 2 Blocks from 6 Train. Cle... The room is open and comfortable. Quiet and sp... Great Neighborhood. 2 Blocks from 6 Train. Cle... none Around the Corner on 111th (btw 2nd & 3rd) the... ... None None f strict f f 1 0.14 64.780300 4.780300
24 1072481 https://www.airbnb.com/rooms/1072481 20161203092247 2016-12-04 Bedroom for two in Chelsea Clean, safe, and, most of all, inexpensive roo... Clean, safe, and, most of all, inexpensive roo... none None ... None None f strict t t 1 2.48 129.715260 49.715260
25 3431816 https://www.airbnb.com/rooms/3431816 20161203092247 2016-12-04 APT IN PROSPECT/CROWN HEIGHTS 2,3,4,5 - S shuttle, B,Q subway access from th... This apartment is a bright, one bedroom, on th... 2,3,4,5 - S shuttle, B,Q subway access from th... none The apartment is located in the heart of Crown... ... None None f moderate f f 1 0.11 139.714071 29.714071
26 3038614 https://www.airbnb.com/rooms/3038614 20161203092247 2016-12-04 Quiet*Spacious*Private Bath*Sunny! Modern oasis in a quiet setting near Prospect ... ::: In 2 BR/ 2 Bath Apt ::: *contact me for l... Modern oasis in a quiet setting near Prospect ... none My neighborhood is great because you can walk ... ... None None f flexible f f 1 0.11 80.357121 -19.642879
27 9198992 https://www.airbnb.com/rooms/9198992 20161203092247 2016-12-04 Modern NY apt in the East Village! Our cool and comfortable one bedroom flat with... None Our cool and comfortable one bedroom flat with... none None ... None None f strict f f 1 0.27 173.511245 24.511245
28 528251 https://www.airbnb.com/rooms/528251 20161203092247 2016-12-04 Spacious, Well-Lit Comfortable Apt! Spacious 1-bedroom 9th floor apartment in Pro... Spacious 1-bedroom 9th floor apartment in Pro... none None ... None None f flexible f f 1 0.25 149.324665 19.324665
29 16014657 https://www.airbnb.com/rooms/16014657 20161203092247 2016-12-04 Beautiful private bedroom - LIC This is a very charming private bedroom, in LI... None This is a very charming private bedroom, in LI... none None ... None None t strict f f 1 NaN 59.103421 -7.896579
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
39631 12710376 https://www.airbnb.com/rooms/12710376 20161203092247 2016-12-04 Private Bedroom #5 This lovely room is located in the heart of Qu... None This lovely room is located in the heart of Qu... none None ... None None f moderate f f 6 0.63 76.074273 6.074273
39632 12713863 https://www.airbnb.com/rooms/12713863 20161203092247 2016-12-04 Private Bedroom #2 This bright room is located in the heart of Qu... None This bright room is located in the heart of Qu... none None ... None None f moderate f f 6 2.20 67.208581 12.208581
39633 12713927 https://www.airbnb.com/rooms/12713927 20161203092247 2016-12-04 Private room #3 This newly renovated room is located in the he... None This newly renovated room is located in the he... none None ... None None f moderate f f 6 2.29 70.965539 0.965539
39634 12294124 https://www.airbnb.com/rooms/12294124 20161203092247 2016-12-04 Friendly and Inviting Home These lovely rooms are located in the heart of... None These lovely rooms are located in the heart of... none None ... None None f moderate f f 6 0.70 146.715222 6.715222
39635 12713763 https://www.airbnb.com/rooms/12713763 20161203092247 2016-12-04 Private Bedroom #1 The room is located in the heart of Queens in ... None The room is located in the heart of Queens in ... none None ... None None f moderate f f 6 1.00 41.294453 -8.705547
39636 12713995 https://www.airbnb.com/rooms/12713995 20161203092247 2016-12-03 Private Queen Bed #4 This lovely room is located in the heart of Qu... None This lovely room is located in the heart of Qu... none None ... None None f moderate f f 6 1.21 77.097705 7.097705
39637 14155795 https://www.airbnb.com/rooms/14155795 20161203092247 2016-12-03 Room for rent. Quiet, safe. Clean safe room. Quiet apartment in the middle... Truly in one of the best neighborhoods in NYC Clean safe room. Quiet apartment in the middle... none It's the East Village in NYC. Surrounded by gr... ... None None f flexible f f 1 1.88 102.634572 17.634572
39638 11272265 https://www.airbnb.com/rooms/11272265 20161203092247 2016-12-03 Spacious 3 BR APT in Midtown(5/6AVE Perfect Midtown Manhattan location, 1 min from... It's a big 3bedroom Apt in Midtown - which is ... Perfect Midtown Manhattan location, 1 min from... none Our place is perfectly located in Midtown Manh... ... None None f strict f f 1 1.20 313.583946 -161.416054
39639 6029332 https://www.airbnb.com/rooms/6029332 20161203092247 2016-12-03 Very private & large room Wburg BK Renting my large bedroom in Williamsburg, Broo... I will be out of town for a week in November s... Renting my large bedroom in Williamsburg, Broo... none It's Williamsburg, literally the hippest place... ... None None f flexible f f 1 0.21 93.007897 -16.992103
39640 14371812 https://www.airbnb.com/rooms/14371812 20161203092247 2016-12-04 Comfortable One Private Bedroom near Manhattan This one bedroom is ideal for business people,... None This one bedroom is ideal for business people,... none The neighborhood is safe and located in a conv... ... None None f strict f f 3 NaN 34.587826 -11.412174
39641 14549056 https://www.airbnb.com/rooms/14549056 20161203092247 2016-12-03 LIC/Astoria Two Bed Apt 10 minutes from Manhattan This two bedroom apartment is ideal for busine... TRANSPORTATION: The apartment is located on a ... This two bedroom apartment is ideal for busine... none The neighborhood is a vibrant, multicultural c... ... None None f strict f f 3 0.95 146.284377 16.284377
39642 13926463 https://www.airbnb.com/rooms/13926463 20161203092247 2016-12-03 Lovely Private room near Manhattan One comfy, bright and intimate bedroom availab... ABOUT THE APARTMENT: This great third floor a... One comfy, bright and intimate bedroom availab... none The neighborhood is a vibrant, multicultural c... ... None None f strict f f 3 3.91 41.500310 -28.499690
39643 11927624 https://www.airbnb.com/rooms/11927624 20161203092247 2016-12-03 Light filled modern APT in DUMBO Come visit our beautiful light-filled apartmen... We have two bedrooms with a pullout full size ... Come visit our beautiful light-filled apartmen... none The neighborhood is the best neighborhood in N... ... None None f strict f f 1 0.38 231.387627 61.387627
39644 15073292 https://www.airbnb.com/rooms/15073292 20161203092247 2016-12-03 Private Studio; Near All NYC Small Studio with private entrance and private... Queen Sofa bed and pull out twin bed Small Studio with private entrance and private... none Grocery Stores, Bars, Italian Restaurants, CVS... ... None None f flexible f f 1 1.00 142.895464 42.895464
39645 11617806 https://www.airbnb.com/rooms/11617806 20161203092247 2016-12-03 Bedroom in Williamsburg, BK, NYC Cozy bedroom in shared apartment in Williamsbu... Bright sunny room in Williamsburg! Our buildin... Cozy bedroom in shared apartment in Williamsbu... none None ... None None f flexible f f 2 1.20 97.699610 52.699610
39646 12189731 https://www.airbnb.com/rooms/12189731 20161203092247 2016-12-03 Very bright room in 3BD/2BR Wall-sized window, 5 mins from Manhattan throu... Very bright room in prime Williamsburg. It has... Wall-sized window, 5 mins from Manhattan throu... none None ... None None f flexible f f 2 0.26 107.164332 47.164332
39647 11010126 https://www.airbnb.com/rooms/11010126 20161203092247 2016-12-03 Gramercy: Convenient Bright Quiet My place is a sanctuary. It is pristine, quie... One single bed. Hardwood Floors. Windows in ev... My place is a sanctuary. It is pristine, quie... none The apartment is blocks from the water, subway... ... None None f flexible f f 1 NaN 103.030304 -21.969696
39648 15708092 https://www.airbnb.com/rooms/15708092 20161203092247 2016-12-03 Charming 1BR A in townhouse with modern amenities Spacious & bright.2 blocks/3 mins to express t... A conveniently-located, spacious and comfortab... Spacious & bright.2 blocks/3 mins to express t... none From its inception, Harlem has kept a cadence ... ... None None f strict t t 3 1.00 114.335475 28.335475
39649 15675757 https://www.airbnb.com/rooms/15675757 20161203092247 2016-12-03 Charming 1BR B in townhouse with modern amenities Spacious & bright.2 blocks/3 mins to express t... A conveniently-located, spacious and comfortab... Spacious & bright.2 blocks/3 mins to express t... none From its inception, Harlem has kept a cadence ... ... None None f flexible t t 3 NaN 107.006962 14.006962
39650 15859751 https://www.airbnb.com/rooms/15859751 20161203092247 2016-12-03 Charming 2BR in townhouse with modern amenities Spacious & bright.2 blocks/3 mins to express t... A conveniently-located, spacious and comfortab... Spacious & bright.2 blocks/3 mins to express t... none rom its inception, Harlem has kept a cadence l... ... None None f strict t t 3 NaN 221.469670 -23.530330
39651 14297551 https://www.airbnb.com/rooms/14297551 20161203092247 2016-12-03 Home Away from Home You’ll love my place because of the comfy bed,... None You’ll love my place because of the comfy bed,... none None ... None None t flexible f f 1 0.27 73.005905 23.005905
39652 15072707 https://www.airbnb.com/rooms/15072707 20161203092247 2016-12-04 Holiday in Heart of Time Square/NYC Elevator/Doorman building in the Heart of Time... This is a super cool brand new unit- it's a tw... Elevator/Doorman building in the Heart of Time... none Heart of Time Square, middle of Manhattan in t... ... None None f strict f f 3 4.34 151.027216 11.027216
39653 15073845 https://www.airbnb.com/rooms/15073845 20161203092247 2016-12-03 Brandnew Luxury flat in Time Square Elevator/Doorman building in the Heart of Time... This is a super cool brand new unit- it's a tw... Elevator/Doorman building in the Heart of Time... none Heart of Time Square, middle of Manhattan in t... ... None None f flexible f f 3 3.20 158.962897 18.962897
39654 15073307 https://www.airbnb.com/rooms/15073307 20161203092247 2016-12-03 2bdr Flat - Xmass in Time Squre NYC Elevator/Doorman building in the Heart of Time... This is a super cool brand new unit- it's a tw... Elevator/Doorman building in the Heart of Time... none Heart of Time Square, middle of Manhattan in t... ... None None f strict f f 3 NaN 304.922194 24.922194
39655 6422551 https://www.airbnb.com/rooms/6422551 20161203092247 2016-12-03 cute and clean in trendy Bushwick Cute and clean bedroom with lots of sun and gr... This is a cute bedroom with amazing, happy lig... Cute and clean bedroom with lots of sun and gr... none Bushwick is now referred to as "the hippest ne... ... None None f flexible f f 1 NaN 79.204960 4.204960
39656 9118958 https://www.airbnb.com/rooms/9118958 20161203092247 2016-12-03 Big room in private house Close to all transportations , 25 minutes from... This place is clean , quiet , near to the nice... Close to all transportations , 25 minutes from... none Quiet , safe , near to all ... None None f flexible f f 3 0.13 47.797366 -17.202634
39657 5309959 https://www.airbnb.com/rooms/5309959 20161203092247 2016-12-03 A room in private house, small room Small room for one person nice and clean , nea... Peaceful and clean Access to the kitchen for ... Small room for one person nice and clean , nea... none Very independent and safe , near to a beautifu... ... None None f flexible f f 3 1.16 57.516149 7.516149
39658 11228502 https://www.airbnb.com/rooms/11228502 20161203092247 2016-12-03 A nice room in private house A room in private house near to all transporta... The place is very clean , near to all transpor... A room in private house near to all transporta... none The place is quiet near to a beautiful park ( ... ... None None f flexible f f 3 NaN 57.482650 -7.517350
39659 15859741 https://www.airbnb.com/rooms/15859741 20161203092247 2016-12-03 Home away from home in heart of BK A home away from home to return to after a day... Your home away from home is a beautifully deco... A home away from home to return to after a day... none Lefferts Gardens (also known as Prospect Leffe... ... None None f moderate f f 1 NaN 49.761213 -44.238787
39660 14024708 https://www.airbnb.com/rooms/14024708 20161203092247 2016-12-03 Your beach-side bungalow awaits! You’ll love my place because of the coziness a... Hard-wood floors, custom paint colors, neat an... You’ll love my place because of the coziness a... none It is a very small, tight-knot community. The ... ... None None t strict f f 1 NaN 78.260990 16.260990

39661 rows × 97 columns

Take the predictions


In [14]:
one_listing = train.loc[1, ["preds", "diff", "price"]]

In [119]:
import matplotlib.mlab as mlab
from bokeh.plotting import figure, output_file, show

In [120]:
mu = one_listing.preds
sigma = 50
x = np.linspace(mu-3*sigma, mu+3*sigma, 100)
dist = mlab.normpdf(x, mu, sigma)

In [138]:
def get_normal_pdf(mu, sigma):
    x = np.linspace(mu-3*sigma, mu+3*sigma, 100)
    dist = mlab.normpdf(x, mu, sigma)
    return (x, dist)

In [139]:
p = figure(title="distributions", plot_width=600, plot_height=300)
p.line(x,dist)
p.ray(x=[one_listing.price], y=[0], length=0, angle=np.pi/2, line_width=2, color = "red")



show(p)



In [145]:
def get_normal_pdf(mu, sigma):
    x = np.linspace(mu-3*sigma, mu+3*sigma, 100)
    dist = mlab.normpdf(x, mu, sigma)
    return (x, dist)

def get_price_plot(one_listing, std = 50):
    x, dist = get_normal_pdf(one_listing.preds, std)
    
    p = figure(title="Price Distribution for Similar Listings. Red line is actual price.", plot_width=600, plot_height=300)
    p.line(x,dist)
    p.ray(x=[one_listing.price], y=[0], length=0, angle=np.pi/2, line_width=2, color = "red")
    
    return(p)

In [143]:
plot = get_price_plot(one_listing, 100)

In [144]:
show(plot)



In [ ]: